package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"aqielife.cn/spider/db"
	"aqielife.cn/spider/fake"
	"aqielife.cn/spider/utils"
)

var (
	kds = []string{
		"golang",
	}
	citys = []string{
		"北京",
		"上海",
		"广州",
		"深圳",
		"杭州",
		"成都",
	}

	initResults = []InitResult{}
	loopResults = []LoopResult{}
	jobPipeline = NewJobPipeline()

	wg sync.WaitGroup
)

// downloader start
var (
	jobsApiUrl = "https://www.lagou.com/jobs/positionAjax.json?city=%s&needAddtionalResult=false"
)

type ListResult struct {
	Code    int
	Success bool
	Msg     string
	Content Content
}

type Content struct {
	PositionResult PositionResult
	PageNo         int
	PageSize       int
}

type PositionResult struct {
	Result     []Result
	TotalCount int
}

type Result struct {
	City              string
	BusinessZones     []string
	CompanyFullName   string
	CompanyLabelList  []string
	CompanyShortName  string
	CompanySize       string
	CreateTime        string
	District          string
	Education         string
	FinanceStage      string
	FirstType         string
	IndustryField     string
	IndustryLables    []string
	JobNature         string
	Latitude          string
	Longitude         string
	PositionAdvantage string
	PositionId        int32
	PositionLables    []string
	PositionName      string
	Salary            string
	SecondType        string
	Stationname       string
	Subwayline        string
	Linestaion        string
	WorkYear          string
}

type jobService struct {
	City string
}

func NewJobService(city string) *jobService {
	return &jobService{City: city}
}

func (l *jobService) GetUrl() string {
	req := fmt.Sprintf(jobsApiUrl, l.City)
	url, _ := url.Parse(req)
	query := url.Query()
	url.RawQuery = query.Encode()

	return url.String()
}

func (l *jobService) GetJobs(pn int, kd string) (*ListResult, error) {
	//client := fake.ProxyAuth{License: "", SecretKey: ""}.GetProxyClient()
	client := http.Client{}
	postReader := strings.NewReader(fmt.Sprintf("first=false&pn=%d&kd=%s", pn, kd))
	req, err := http.NewRequest("POST", l.GetUrl(), postReader)
	if err != nil {
		log.Printf("http.NewRequest err: %v", err)
	}

	//req.Header.Set("Proxy-Switch-Ip", "yes")

	req.Header.Add("Accept", "application/json, text/javascript, */*; q=0.01")
	req.Header.Add("Accept-Encoding", "gzip, deflate, br")
	req.Header.Add("Accept-Languag", "zh-CN,zh;q=0.9")
	req.Header.Add("Connection", "keep-alive")
	req.Header.Add("Content-Length", "25")
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
	req.Header.Add("Cookie", "_ga=GA1.2.161331334.1522592243; "+
		"user_trace_token=20180401221723-"+utils.GetUUID()+"; "+
		"LGUID=20180401221723-"+utils.GetUUID()+"; "+
		"index_location_city=%E6%B7%B1%E5%9C%B3; "+
		"JSESSIONID="+utils.GetUUID()+"; "+
		"_gid=GA1.2.1140631185.1523090450; "+
		"Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1522592243,1523090450; "+
		"TG-TRACK-CODE=index_search; _gat=1; "+
		"LGSID=20180407221340-"+utils.GetUUID()+"; "+
		"PRE_UTM=; PRE_HOST=; PRE_SITE=https%3A%2F%2Fwww.lagou.com%2F; "+
		"PRE_LAND=https%3A%2F%2Fwww.lagou.com%2Fjobs%2Flist_golang%3FlabelWords%3D%26fromSearch%3Dtrue%26suginput%3D; "+
		"Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1523110425; "+
		"LGRID=20180407221344-"+utils.GetUUID()+"; "+
		"SEARCH_ID="+utils.GetUUID()+"")
	req.Header.Add("Host", "www.lagou.com")
	req.Header.Add("Origin", "https://www.lagou.com")
	req.Header.Add("Referer", "https://www.lagou.com/jobs/list_golang?labelWords=&fromSearch=true&suginput=")
	req.Header.Add("User-Agent", fake.GetUserAgent())

	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var results ListResult
	err = json.Unmarshal([]byte(body), &results)
	if err != nil {
		return nil, err
	}

	return &results, nil
}

// downloader end

// model start
var (
	jobs []LgJob
)

type LgJob struct {
	// 城市名称
	City string
	// 地区
	District string

	// 公司简称
	CompanyShortName string
	// 公司全称
	CompanyFullName string
	// 公司标签
	CompanyLabelList string
	// 公司规模
	CompanySize string
	// 融资阶段
	FinanceStage string

	// 行业领域
	IndustryField string
	// 行业标签
	IndustryLables string

	// 职位名称
	PositionName string
	// 职位标签
	PositionLables string
	// 职位诱惑
	PositionAdvantage string
	// 工作年限
	WorkYear string
	// 学历要求
	Education string
	// 薪资范畴
	Salary string

	// 经度
	Longitude float64
	// 纬度
	Latitude float64
	// 附近的地铁
	Linestaion string

	// 发布时间
	CreateTime int64
	// 新增时间
	AddTime int64
}

func NewJobPipeline() *LgJob {
	return &LgJob{}
}

func (j *LgJob) Append(js []LgJob) {
	mutex.Lock()
	jobs = append(jobs, js...)
	mutex.Unlock()
}

func (j *LgJob) Get() []LgJob {
	return jobs
}

func (j *LgJob) Push() error {
	for index, v := range j.Get() {
		if err := db.DB.Create(&v).Error; err != nil {
			log.Printf("db.Create err : %d, %v", index, err)
		} else {
			log.Println("新增成功: ", index)
		}
	}

	return nil
}

// model end

// scheduler start

var (
	mutex     sync.Mutex
	JobParams = []JobParam{}
)

type JobParam struct {
	City string
	Pn   int
	Kd   string
}

func NewJobScheduler() *JobParam {
	return &JobParam{}
}

func (j *JobParam) Pop() *JobParam {
	mutex.Lock()
	length := len(JobParams)
	if length < 1 {
		mutex.Unlock()
		return nil
	}

	job := JobParams[length-1]
	JobParams = JobParams[:length-1]

	mutex.Unlock()
	return &job
}

func (j *JobParam) Append(city string, pn int, kd string) {
	mutex.Lock()
	JobParams = append(JobParams, JobParam{
		City: city,
		Pn:   pn,
		Kd:   kd,
	})

	mutex.Unlock()
}

func (j *JobParam) Count() int {
	return len(JobParams)
}

// scheduler end

// convert start
func ToPipelineJobs(dJobs []Result) []LgJob {
	var pJobs []LgJob
	for _, v := range dJobs {
		longitude, _ := strconv.ParseFloat(v.Longitude, 64)
		latitude, _ := strconv.ParseFloat(v.Latitude, 64)
		pJobs = append(pJobs, LgJob{
			City:     v.City,
			District: v.District,

			CompanyShortName: v.CompanyShortName,
			CompanyFullName:  v.CompanyFullName,
			CompanyLabelList: strings.Join(v.CompanyLabelList, ","),
			CompanySize:      v.CompanySize,
			FinanceStage:     v.FinanceStage,

			PositionName:      v.PositionName,
			PositionLables:    strings.Join(v.PositionLables, ","),
			PositionAdvantage: v.PositionAdvantage,
			WorkYear:          v.WorkYear,
			Education:         v.Education,
			Salary:            v.Salary,

			IndustryField:  v.IndustryField,
			IndustryLables: strings.Join(v.IndustryLables, ","),

			Longitude:  longitude,
			Latitude:   latitude,
			Linestaion: v.Linestaion,

			CreateTime: utils.MustDateToUnix(v.CreateTime),
			AddTime:    time.Now().Unix(),
		})
	}

	return pJobs
}

// convert end

// spider start
var (
	delayTime    = time.Tick(time.Millisecond * 500)
	jobScheduler = NewJobScheduler()
)

type InitResult struct {
	City       string
	Kd         string
	TotalPage  int
	TotalCount int
}

type LoopResult struct {
	Success int
	Error   int
	Empty   int
	Errors  []string
}

func InitJobs(city string, pn int, kd string) ([]InitResult, error) {
	var (
		jobs       []Result
		totalPage  int
		totalCount int
		results    []InitResult

		err error
	)

	jobs, totalPage, totalCount, err = GetJobs(city, pn, kd)
	if err != nil {
		return nil, err
	}

	results = append(results, InitResult{
		City:       city,
		Kd:         kd,
		TotalPage:  totalPage,
		TotalCount: totalCount,
	})

	for i := 2; i <= totalPage; i++ {
		jobScheduler.Append(city, i, kd)
	}

	jobPipeline.Append(ToPipelineJobs(jobs))

	return results, nil
}

func LoopJobs() LoopResult {
	var (
		result LoopResult
		output = jobScheduler.Count()

		params = make(chan []Result)
	)

	for i := 0; i < output; i++ {
		<-delayTime
		go func() {
			if jobParam := jobScheduler.Pop(); jobParam != nil {
				jobs, _, _, err := GetJobs(jobParam.City, jobParam.Pn, jobParam.Kd)
				if err != nil {
					result.Error++
					result.Errors = append(result.Errors, err.Error())
				} else {
					params <- jobs
				}
			} else {
				result.Empty++
			}
		}()
	}

L:
	for {
		select {
		case p := <-params:
			result.Success++
			jobPipeline.Append(ToPipelineJobs(p))
		default:
			if (result.Success + result.Error + result.Empty) >= output {
				log.Printf("Break...")
				break L
			}
		}
	}

	return result
}

func GetJobs(city string, pn int, kd string) ([]Result, int, int, error) {
	totalPage := 0
	jobService := NewJobService(city)
	result, err := jobService.GetJobs(pn, kd)
	if err != nil {
		return nil, 0, 0, err
	}

	log.Printf("GetJobs Code: %d, GetJobs City: %s, Pn: %d, Kd: %s", result.Code, city, pn, kd)

	if result.Code == 0 && result.Success == true {
		content := result.Content
		if content.PositionResult.TotalCount > 0 && content.PageSize > 0 {
			totalPage = utils.CalculateTotalPage(float64(content.PositionResult.TotalCount), float64(content.PageSize))
		}
	} else {
		return nil, 0, 0, errors.New(fmt.Sprintf("GetJobs City: %s, Pn: %d, Kd: %s, Result: %v", city, pn, kd, result))
	}

	return result.Content.PositionResult.Result, totalPage, result.Content.PositionResult.TotalCount, nil
}

// spider end

func main() {
	for _, kd := range kds {
		for _, city := range citys {
			wg.Add(1)
			go func(city string, kd string) {
				defer wg.Done()
				initResult, err := InitJobs(city, 1, kd)
				if err != nil {
					log.Fatalln(err)
				}

				initResults = append(initResults, initResult...)
				loopResults = append(loopResults, LoopJobs())
			}(city, kd)
		}
	}

	wg.Wait()

	jobPipeline.Push()

	log.Printf("Init Results: %v", initResults)
	log.Printf("Loop Results: %v", loopResults)
}
